In [1]:
#package
import pandas as pd
import missingno as msno
import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots
from statistics import median
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
In [2]:
# Load data
path = 'D:/USA 2021 Spring/Data_analysis/train.csv'
traindf = pd.read_csv(path)
In [3]:
path = 'D:/USA 2021 Spring/Data_analysis/users.csv'
userdf = pd.read_csv(path)
In [4]:
path = 'D:/USA 2021 Spring/Data_analysis/test.csv'
testdf = pd.read_csv(path)
In [5]:
traindf = pd.merge(traindf, userdf, on = 'user_id')
testdf = pd.merge(testdf, userdf, on = 'user_id')
display(traindf.head(), testdf.head())
Step 1 Exploration and Data Clean¶
Data information¶
In [6]:
traindf.info()
In [7]:
traindf.describe()
Out[7]:
In [8]:
def get_summary_df(df):
columns = df.columns.tolist()
dtypes = []
unique_counts = []
missing_counts = []
missing_percentages = []
total_counts = [df.shape[0]] * len(columns)
for column in columns:
dtype = str(df[column].dtype)
dtypes.append(dtype)
unique_count = df[column].nunique()
unique_counts.append(unique_count)
missing_count = df[column].isnull().sum()
missing_counts.append(missing_count)
missing_percentage = round((missing_count/df.shape[0]) * 100, 2)
missing_percentages.append(missing_percentage)
summary_df = pd.DataFrame({
"column": columns,
"dtype": dtypes,
"unique_count": unique_counts,
"missing_count": missing_counts,
"missing_percentage": missing_percentages,
"total_count": total_counts,
})
summary_df = summary_df.sort_values(by="missing_percentage", ascending=False).reset_index(drop=True)
return summary_df
In [9]:
get_summary_df(traindf)
Out[9]:
In [10]:
for col in traindf.columns:
if traindf[col].dtype == "object":
print (traindf[col].unique())
In [11]:
#date
traindf['grass_date'] = pd.to_datetime(traindf['grass_date']).dt.date
testdf['grass_date'] = pd.to_datetime(testdf['grass_date']).dt.date
In [12]:
traindf['year'] = pd.to_datetime(traindf['grass_date']).dt.year
traindf['month'] = pd.to_datetime(traindf['grass_date']).dt.month
traindf['day'] = pd.to_datetime(traindf['grass_date']).dt.day
testdf['year'] = pd.to_datetime(testdf['grass_date']).dt.year
testdf['month'] = pd.to_datetime(testdf['grass_date']).dt.month
testdf['day'] = pd.to_datetime(testdf['grass_date']).dt.day
In [13]:
#as type
traindf['last_open_day'] = traindf['last_open_day'].replace('Never open', 0)
traindf['last_login_day'] = traindf['last_login_day'].replace('Never login', 0)
traindf['last_checkout_day'] = traindf['last_checkout_day'].replace('Never checkout', 0)
testdf['last_open_day'] = testdf['last_open_day'].replace('Never open', 0)
testdf['last_login_day'] = testdf['last_login_day'].replace('Never login', 0)
testdf['last_checkout_day'] = testdf['last_checkout_day'].replace('Never checkout', 0)
In [14]:
for col in ['last_open_day','last_login_day','last_checkout_day']:
traindf[col] = traindf[col].astype(np.int32)
In [15]:
for col in ['last_open_day','last_login_day','last_checkout_day']:
testdf[col] = testdf[col].astype(np.int32)
In [16]:
#missing variable
msno.matrix(traindf)
Out[16]:
In [17]:
traindf['attr_1'].fillna(2.0, inplace = True)
traindf['attr_2'].fillna(traindf['attr_2'].value_counts().index[0], inplace=True)
In [18]:
testdf['attr_1'].fillna(2.0, inplace = True)
testdf['attr_2'].fillna(testdf['attr_2'].value_counts().index[0], inplace=True)
In [19]:
import statistics
median = round(traindf['age'].median())
std = traindf['age'].std()
outliers = (traindf['age']-median).abs()>std
traindf['age'][outliers]=np.nan
traindf['age'].fillna(median, inplace=True)
In [20]:
median = round(testdf['age'].median())
std = testdf['age'].std()
outliers = (testdf['age']-median).abs()>std
testdf['age'][outliers]=np.nan
testdf['age'].fillna(median, inplace=True)
In [21]:
msno.matrix(traindf)
Out[21]:
Target variables¶
In [22]:
#imbalance
fig = px.histogram(traindf, x="open_flag")
fig.show()
In [23]:
traindf['open_flag'].value_counts(normalize = True)
Out[23]:
Input variables¶
In [24]:
del traindf['grass_date'], testdf['grass_date'], traindf['user_id'], testdf['user_id'], traindf['row_id'], testdf['row_id']
In [25]:
cat_features = ['country_code', 'domain', 'year', 'month', 'day', 'attr_1', 'attr_2', 'attr_3']
num_features = [col for col in traindf.columns if col != "open_flag" and col not in cat_features]
print(cat_features, num_features)
In [26]:
#continuous variable
def plot_feature_scatter(df1, df2, features):
i = 0
sns.set_style("whitegrid")
plt.figure
fig, ax = plt.subplots(4, 4, figsize=(20, 20))
for feature in features:
i += 1
plt.subplot(4, 4, i)
plt.scatter(df1[feature], df2[feature], marker="+", color='#2B3A67', alpha=0.2)
plt.xlabel(feature, fontsize=9)
plt.show()
In [27]:
#scalization. normalization
plot_feature_scatter(traindf.sample(50000), testdf.sample(50000), num_features)
In [28]:
plt.figure(figsize=(14,10))
plt.title('Pearson Correlation of Features', y=1.05, size=15)
sns.heatmap(traindf[num_features].corr(), cmap = "BrBG",
linewidths=0.1, square=True, linecolor='white',
annot=True)
plt.show()
In [29]:
def plot_feature_distribution(df1, df2, label1, label2, features, size=[3,5]):
i = 0
sns.set_style("whitegrid")
plt.figure()
fig, ax = plt.subplots(size[0], size[1], figsize=(22, 20))
for feature in features:
i += 1
plt.subplot(size[0], size[1], i)
sns.distplot(df1[feature], hist=False, label=label1, kde_kws={'bw':1.5}, color="#5982C5")
sns.distplot(df2[feature], hist=False, label=label2, kde_kws={'bw':1.5}, color="#FB3523")
plt.xlabel(feature, fontsize=9)
locs, labels = plt.xticks()
plt.tick_params(axis="x", which="major", labelsize=6, pad=6)
plt.tick_params(axis="y", which="major", labelsize=6)
plt.show()
In [30]:
target_0_df = traindf.loc[traindf["open_flag"] == 0]
target_1_df = traindf.loc[traindf["open_flag"] == 1]
plot_feature_distribution(target_0_df, target_1_df, '0', '1', num_features, size=[3,5])
In [31]:
#category variable
lbls, freqs = np.unique(traindf['country_code'].values, return_counts = True)
plt.figure(figsize = (10, 10))
plt.title('Train - Country Code')
plt.pie(freqs, labels = lbls, autopct = '%1.1f%%', shadow = False, startangle = 90)
plt.show()
In [32]:
target = traindf['open_flag'].copy()
del traindf['open_flag']
target
Out[32]:
In [33]:
get_summary_df(traindf)
Out[33]:
Step 2 Modeling¶
In [34]:
from sklearn.model_selection import train_test_split
# Train_Test Split
Xtrain, Xvalid, ytrain, yvalid = train_test_split(traindf,target,test_size=0.30)
In [35]:
ytrain
Out[35]:
In [36]:
#preprocessing
from sklearn.compose import make_column_transformer
from sklearn.preprocessing import StandardScaler,OneHotEncoder, MinMaxScaler
preprocessor = make_column_transformer(
(OneHotEncoder(), cat_features),
(MinMaxScaler(), num_features),
remainder='passthrough'
)
#preprocessor1 = make_column_transformer(
# (OneHotEncoder(), cat_features),
# (StandardScaler(), num_features),
# remainder='passthrough'
#)
In [37]:
#use the model to deal with the imbalance
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LogisticRegression, SGDClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.naive_bayes import BernoulliNB
from sklearn.neural_network import MLPClassifier
from sklearn.ensemble import RandomForestClassifier
import xgboost as xgb
import lightgbm as lgb
models = []
#linear model
models.append(("simple_logistic",
make_pipeline(preprocessor,
LogisticRegression())))
models.append(("simple_logistic_elaticnet",
make_pipeline(preprocessor,
SGDClassifier(loss='log', penalty='elasticnet', alpha=0.0001, l1_ratio=0.15))))
models.append(("naivebayes",
make_pipeline(preprocessor,
BernoulliNB())))
#non-linear model
models.append(("random_forest",
make_pipeline(preprocessor,
RandomForestClassifier(max_depth=2, random_state=0))))
models.append(("xgboost",
make_pipeline(preprocessor,
xgb.XGBClassifier())))
models.append(("lgboost",
make_pipeline(preprocessor,
lgb.LGBMClassifier())))
In [38]:
from sklearn.metrics import matthews_corrcoef, make_scorer
from sklearn.model_selection import RepeatedKFold, cross_validate
from sklearn.metrics import matthews_corrcoef
#k-fold
my_ctrl = RepeatedKFold(n_splits=5, n_repeats=2)
#performance matrix
mcc = make_scorer(matthews_corrcoef)
my_metric =(mcc)
In [39]:
from numpy import absolute, mean, std
model_results = []
names = []
for name, model in models:
cv_results = cross_validate(model, Xtrain, ytrain, scoring=my_metric, cv=my_ctrl, n_jobs=-1)
model_results.append(cv_results["test_score"])
names.append(name)
msg = "%s:(%s) %f (%f)" % (name, model[-1], mean(absolute(model_results)), std(absolute(model_results)))
print(msg)
In [40]:
Mcc = pd.DataFrame(absolute(model_results),index=names).T
In [41]:
import plotly.graph_objects as go
fig = go.Figure()
for name in names:
fig.add_trace(go.Box(x=Mcc[name],name=name))
fig.update_layout(title_text="Mcc")
fig.show()
In [42]:
Xprocess = preprocessor.fit_transform(Xtrain)
In [43]:
from hyperopt import hp, fmin, tpe, Trials, space_eval
from hyperopt.pyll.base import scope
import time
from sklearn.model_selection import cross_val_score
# objective function
def ho_func(params):
estimator = lgb.LGBMClassifier(objective='binary',
boosting_type='goss',
is_unbalance=False,
learning_rate=0.03,
metric='roc_auc,log_loss', n_estimators=200,
class_weight=params['class_weight'],
num_leaves=31, random_state=42, max_depth=-1, reg_lambda=5, reg_alpha=1)
estimator.fit(Xprocess, ytrain)
# pred_array = estimator.predict(X_test)
# res = accuracy_score(y_test, pred_array)
# pred_test = estimator.predict(X_test)
# res = roc_auc_score(y_test, pred_test)
res = cross_val_score(estimator, Xprocess, ytrain,
scoring=mcc, cv=3).mean()
print("*" * 30)
print(params)
print("cv mcc: {}".format(res))
return 1-res
In [44]:
weight_list = [{0: one} for one in list(np.arange(0, 10, 0.1))]
ho_space = {
'max_depth': hp.choice('max_depth', range(5, 30, 1)),
# 'learning_rate' : hp.quniform('learning_rate', 0.01, 0.5, 0.01),
'n_estimators': hp.choice('n_estimators', range(100, 1000, 100)),
# 'subsample' : hp.quniform('subsample', 0.1, 1, 0.01),
# 'colsample_bytree' : hp.quniform('colsample_bytree', 0.1, 1.0, 0.01),
# 'min_data_in_leaf': hp.choice('min_data_in_leaf', range(3, 15, 2)),
'reg_alpha': hp.quniform('reg_alpha', 0, 3,0.1),
'reg_lambda': hp.quniform('reg_lambda', 0, 5,0.1),
'num_leaves': hp.choice('num_leaves', range(20, 50, 1)),
'class_weight': hp.choice('class_weight', weight_list), 'verbose': -1}
start=time.time()
trials=Trials()
best=fmin(fn=ho_func, space=ho_space, algo=tpe.suggest,
max_evals=100, trials=trials)
print(f'Total time : {(time.time()-start):.5f} seconds')
print(f'Best Parameters - val_loss:{np.asarray(trials.losses(), dtype=float).max()} for {space_eval(ho_space,best)}')
lbm_model=lgb.LGBMClassifier().set_params(**space_eval(ho_space, best))
lbm_model
Out[44]:
In [45]:
models.append(("lgboost_tune",
make_pipeline(preprocessor,
lbm_model)))
models[-1]
Out[45]:
In [46]:
models[-1][1].fit(Xtrain, ytrain)
Out[46]:
In [47]:
models[-1][1].predict(Xvalid)
Out[47]:
In [48]:
matthews_corrcoef(yvalid, models[-1][1].predict(Xvalid))
Out[48]:
submit¶
In [49]:
testdf.head()
Out[49]:
In [50]:
predictions = models[-1][1].predict(testdf)
In [51]:
path = 'D:/USA 2021 Spring/Data_analysis/sample_submission_0_1.csv'
sampledf = pd.read_csv(path)
sampledf
Out[51]:
In [52]:
sampledf['open_flag'] = predictions
sampledf.to_csv('D:/USA 2021 Spring/Data_analysis/sample_sub_ShopeeEmail.csv', index = False)
sampledf
Out[52]:
In [53]:
fig = px.histogram(sampledf, x="open_flag")
fig.show()
In [54]:
sampledf['open_flag'].value_counts(normalize = True)
Out[54]: